{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RSRVQq1bVmFI"
   },
   "source": [
    "# Все действия были проделаны в google colaboratory на бесплатном GPU\n",
    "  \n",
    "<b> Скачивание файлов с google drive из папки DMIA лежащей в корневой папке, их перемещение и установка gensim\n",
    "  https://drive.google.com/file/d/1fZX_Yu0iD9gAn4lug6m29Fk0-SjF82QK/view?usp=sharing - dataset\n",
    "  http://rusvectores.org/static/models/rusvectores4/fasttext/araneum_none_fasttextskipgram_300_5_2018.tgz - fasttext\n",
    "  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 1024
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 46936,
     "status": "ok",
     "timestamp": 1524240874336,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "TD_jv6jnN4YL",
    "outputId": "11d33d36-b2c4-41fc-c804-de98a692ff24"
   },
   "outputs": [],
   "source": [
    "from google.colab import auth\n",
    "from googleapiclient.discovery import build\n",
    "import io , requests, os\n",
    "import sys\n",
    "auth.authenticate_user()\n",
    "from googleapiclient.discovery import build\n",
    "drive_service = build('drive', 'v3')\n",
    "\n",
    "def get_parent_folder(folder_name):\n",
    "  page_token = None\n",
    "  folder_array = []\n",
    "  query = \"name='%s' and mimeType='application/vnd.google-apps.folder'\" % folder_name\n",
    "  while True:\n",
    "      response = drive_service.files().list(q=query,\n",
    "                                          spaces='drive',\n",
    "                                          fields='nextPageToken,files(id, name)',\n",
    "                                          pageToken=page_token).execute()\n",
    "      for file in response.get('files', []):\n",
    "          # Process change\n",
    "          #print (file.get('name'), file.get('id'))\n",
    "          folder_array.append({\"name\" : file.get('name'), \"id\" : file.get('id')})\n",
    "      page_token = response.get('nextPageToken', None)\n",
    "      if page_token is None:\n",
    "          break\n",
    "  return folder_array\n",
    "\n",
    "def get_files_from_parent(parent_id):\n",
    "  page_token = None\n",
    "  folder_array = dict()\n",
    "  query = \"'%s' in parents\" % parent_id\n",
    "  while True:\n",
    "      response = drive_service.files().list(q=query,\n",
    "                                          spaces='drive',\n",
    "                                          fields='nextPageToken, files(id, name)',\n",
    "                                          pageToken=page_token).execute()\n",
    "      for file in response.get('files', []):\n",
    "          # Process change\n",
    "          #print (file.get('name'), file.get('id'))\n",
    "          folder_array.update({file.get('name'):file.get('id')})\n",
    "      page_token = response.get('nextPageToken', None)\n",
    "      if page_token is None:\n",
    "          break\n",
    "  return folder_array\n",
    "\n",
    "def get_file_buffer(file_id, verbose=0):\n",
    "  from googleapiclient.http import MediaIoBaseDownload\n",
    "  request = drive_service.files().get_media(fileId=file_id)\n",
    "  downloaded = io.BytesIO()\n",
    "  downloader = MediaIoBaseDownload(downloaded, request)\n",
    "  done = False\n",
    "  while done is False:\n",
    "    # _ is a placeholder for a progress object that we ignore.\n",
    "    # (Our file is small, so we skip reporting progress.)\n",
    "    progress, done = downloader.next_chunk()\n",
    "    if verbose:\n",
    "      sys.stdout.flush()\n",
    "      sys.stdout.write('\\r')\n",
    "      percentage_done = progress.resumable_progress * 100/progress.total_size\n",
    "      sys.stdout.write(\"[%-100s] %d%%\" % ('='*int(percentage_done), int(percentage_done)))\n",
    "  downloaded.seek(0)\n",
    "  return downloaded\n",
    "\n",
    "parent_folder = get_parent_folder('DMIA')\n",
    "print(parent_folder)\n",
    "parent_folder[0][\"id\"]\n",
    "input_file_meta = get_files_from_parent(parent_folder[0][\"id\"])\n",
    "print(input_file_meta)\n",
    "for file, id in input_file_meta.items():\n",
    "  downloaded = get_file_buffer(id, verbose=1)\n",
    "  dest_file = os.path.join('.', file)\n",
    "  print(\"processing %s data\" % file)\n",
    "  with open(dest_file, \"wb\") as out:\n",
    "    out.write(downloaded.read())\n",
    "    print(\"Done %s\" % dest_file)\n",
    "\n",
    "!mkdir data\n",
    "!mv araneum_none_fasttextskipgram_300_5_2018.model data/araneum_none_fasttextskipgram_300_5_2018.model\n",
    "!mv train.tsv data/train.tsv\n",
    "!mv public.tsv data/public.tsv\n",
    "!mv araneum_none_fasttextskipgram_300_5_2018.model.syn0_vocab_lockf.npy data/araneum_none_fasttextskipgram_300_5_2018.model.syn0_vocab_lockf.npy\n",
    "!mv araneum_none_fasttextskipgram_300_5_2018.model.syn1neg.npy data/araneum_none_fasttextskipgram_300_5_2018.model.syn1neg.npy\n",
    "!mv araneum_none_fasttextskipgram_300_5_2018.model.wv.syn0.npy data/araneum_none_fasttextskipgram_300_5_2018.model.wv.syn0.npy\n",
    "!mv araneum_none_fasttextskipgram_300_5_2018.model.wv.syn0_vocab.npy data/araneum_none_fasttextskipgram_300_5_2018.model.wv.syn0_vocab.npy\n",
    "\n",
    "!pip install gensim"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "B85w5gU4Nx-l"
   },
   "source": [
    "<b> Описание данных и задачи </b>\n",
    "\n",
    "Каждый эпизод состоит из двух частей – контекста (Context) и финальной реплики (Reply). Например,\n",
    "\n",
    "- context_2: Персонаж A говорит реплику \n",
    "- context_1: Персонаж B отвечает на нее \n",
    "- context_0: Персонаж А произносит вторую реплику \n",
    "- reply: Персонаж B отвечает на вторую реплику \n",
    "\n",
    "Контекстная часть может состоять из трех реплик (как в примере) – в 50% случаев, двух – в 25%, и одного – в оставшихся 25% случаев. Финальная реплика (Reply) всегда завершает любой эпизод, то есть следует за контекстом (Context). Задача участников – найти наиболее подходящую и интересную реплику для данного контекста среди предложенных кандидатов (числом до 6), случайно выбранных из топа кандидатов, возвращенных бейзлайном высокого качества, натренированным командой Алисы (который, в свою очередь, отобрал кандидатов среди всех возможных реплик OpenSubtitles).\n",
    "\n",
    "Все реплики-кандидаты размечены асессорами на сервисе Яндекс.Толока с помощью следующей инструкции для разметки:\n",
    "\n",
    "- Good (2): реплика уместна (имеет смысл для данного контекста) и интересна (нетривиальна, специфична именно для данного контекста, мотивирует продолжать разговор)\n",
    "- Neutral (1): реплика уместна (имеет смысл для данного контекста), но не интересна (тривиальна, не специфична для данного контекста и скорее подталкивает пользователя закончить разговор)\n",
    "- Bad (0): реплика не имеет никакого смысла в данном контексте\n",
    "\n",
    "Каждая метка в тренировочной части датасета (и только в ней), сопровождается также уверенностью (confidence) – числом в интервале от 0 до 1 – которое показывает насколько уверенными в своей разметке были асессоры с Толоки, совместно предложившие данную метку. Мы хотим обратить особое внимание участников на эту информацию, она может быть очень полезна при обучении их моделей."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "UV7lhz4pNx-n"
   },
   "source": [
    "- context_id – идентификатор эпизода\n",
    "- context_2,context_1,context_0 – текст реплик, предшествующих финальной (может состоять из трех частей)\n",
    "- reply_id – идентификатор реплики-кандидата\n",
    "- reply – текст реплики-кандидата\n",
    "- label – метка реплики-кандидата (good, neutral или bad)\n",
    "- confidence - уверенность в метке реплики-кандидата (число от 0 до 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 15242,
     "status": "ok",
     "timestamp": 1524240889642,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "riIwT_myNx-o",
    "outputId": "a48e8011-e984-4bcb-b4b1-2c15ccc10bfd"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from keras.models import Sequential, Model\n",
    "from keras.layers import *\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras import optimizers\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import gensim\n",
    "ft = gensim.models.FastText.load('data/araneum_none_fasttextskipgram_300_5_2018.model')\n",
    "\n",
    "import re\n",
    "\n",
    "from sklearn.metrics import make_scorer, accuracy_score\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "BrG9ub_bNx-w"
   },
   "source": [
    "<b> Метрика для оценки качества. Выбрана исходя из задачи ранжирования."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "DP0V55k_Nx-x"
   },
   "outputs": [],
   "source": [
    "def DCG(label): return sum([float(label[i]/np.log2(i+2)) for i in range(len(label))])\n",
    "\n",
    "def nDCG(label, best_label):\n",
    "    label, best_label = DCG(label), DCG(best_label)\n",
    "    if label != 0 and best_label != 0:\n",
    "        return label/best_label\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "opVW47lENx-0"
   },
   "source": [
    "<b> Загружаем данные, конкатенируем трейн и тест"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "19hn4-XnNx-1"
   },
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(\"data/train.tsv\", sep='\\t', quotechar=' ', header = None)\n",
    "df_train.columns = ['context_id', 'context_2', 'context_1', 'context_0', 'reply_id', 'reply', 'label', 'confidence']\n",
    "\n",
    "df_public = pd.read_csv(\"data/public.tsv\", sep='\\t', quotechar=' ', header = None)\n",
    "df_public.columns = ['context_id', 'context_2', 'context_1', 'context_0', 'reply_id', 'reply']\n",
    "\n",
    "df_data = pd.concat((df_train, df_public)).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ui_lKjh1Nx-5"
   },
   "source": [
    "<b>Заменяем лейблы и заполняем пустые значения confidence на 1 (Почему так рассмотрим далее), остальные на -, меняем порядок столбцов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "rbrmQ46XNx-6"
   },
   "outputs": [],
   "source": [
    "df_data.label = df_data.label.map({'good': 2, 'neutral': 1, 'bad': 0})\n",
    "df_data.confidence.fillna(value=1, inplace=True)\n",
    "df_data.fillna(value='-', inplace=True)\n",
    "df_data = df_data[['context_id', 'context_2', 'context_1', 'context_0', 'reply_id', 'reply', 'label', 'confidence']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nvZzfuf_Nx-9"
   },
   "source": [
    "<b> Выделяем из общего датафрейма трейн и тест (для удобства дальнейшей визуализации)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 117
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1158,
     "status": "ok",
     "timestamp": 1524240894292,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "1kqerVUmNx--",
    "outputId": "0a55ea36-2637-4d36-e3a1-ae7362eb4b62"
   },
   "outputs": [],
   "source": [
    "train_tres = len(df_train)\n",
    "df_train = df_data.loc[:train_tres-1]\n",
    "df_public = df_data.loc[train_tres:]\n",
    "\n",
    "df_train.confidence = df_train.confidence.astype(float)\n",
    "df_train.label = df_train.label.astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cHX4cDZ9Nx_D"
   },
   "source": [
    "# Часть визуализации"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yTQPoAfbNx_D"
   },
   "source": [
    "<b> Смотрим на распределения confidence и лейблов по трейну"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 258
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1041,
     "status": "ok",
     "timestamp": 1524240895427,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "LlCIkRvDNx_E",
    "outputId": "e2a97eba-3de6-4b81-dbe5-163e5698e93d"
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(nrows=1, ncols=2, figsize = (15,3))\n",
    "sns.distplot(df_train['confidence'], bins=100, kde=False, ax=ax[0], color=['b'])\n",
    "sns.countplot(x='label', data=df_train, ax=ax[1]);"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TuTDDYo1Nx_L"
   },
   "source": [
    "<b> Смотрим на распределения кол-ва слов в context и reply в трейн и тесте (слова выделялись пробелами)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 653
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4813,
     "status": "ok",
     "timestamp": 1524240900336,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "I5LLUUQ1Nx_M",
    "outputId": "a0bf8fac-66e4-4220-a498-5d70c0beaf41"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "fig, ax = plt.subplots(nrows=2, ncols=4, figsize=(18,10))\n",
    "\n",
    "for i in range(3):\n",
    "    sns.distplot(df_train[f'context_{2-i}'].apply(lambda x: len(str(x).split(' '))), bins=100, kde=False, ax=ax[0,i])\n",
    "    ax[0,i].set_title('train')\n",
    "sns.distplot(df_train['reply'].apply(lambda x: len(str(x).split(' '))), bins=100, kde=False, ax=ax[0,i+1])\n",
    "ax[0,i+1].set_title('train')\n",
    "\n",
    "for i in range(3):\n",
    "    sns.distplot(df_public[f'context_{2-i}'].apply(lambda x: len(str(x).split(' '))), bins=100, kde=False, ax=ax[1,i])\n",
    "    ax[1,i].set_title('public')\n",
    "sns.distplot(df_public['reply'].apply(lambda x: len(str(x).split(' '))), bins=100, kde=False, ax=ax[1,i+1])\n",
    "ax[1,i+1].set_title('public')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cU-jgjEKNx_R"
   },
   "source": [
    "<b>Как видно из картинок в тесте и трейне кол-ва слов распределены примерно одинаково\n",
    "\n",
    "<b>Далее смотрим на длину всего диалога в тесте и трейне"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 259
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 12552,
     "status": "ok",
     "timestamp": 1524240912909,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "rlsfHD0lNx_S",
    "outputId": "7c23a899-82ea-4ba4-a021-ff78b7e77238"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(18,3))\n",
    "\n",
    "sns.distplot(df_train[['context_2', 'context_1', 'context_0', 'reply']].apply(lambda x : \n",
    "                                                                              len((\" \".join(x.astype(str))).split(' ')), \n",
    "                                                                              axis = 1),\n",
    "            bins=100, kde=False, ax=ax[0])\n",
    "ax[0].set_title('train dialog words length')\n",
    "\n",
    "sns.distplot(df_public[['context_2', 'context_1', 'context_0', 'reply']].apply(lambda x : \n",
    "                                                                              len((\" \".join(x.astype(str))).split(' ')), \n",
    "                                                                              axis = 1),\n",
    "            bins=100, kde=False, ax=ax[1])\n",
    "ax[1].set_title('public dialog words length')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XxPb82VQNx_W"
   },
   "source": [
    "<b>Как и ожидалось распределения одинаковы\n",
    "\n",
    "<b>Теперь подсчитываем из скольки контекстов состоит диалог и смотрим распределения"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 508
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 32190,
     "status": "ok",
     "timestamp": 1524240945129,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "cFQhsNFuNx_X",
    "outputId": "0a55e2df-7673-47ec-cca7-85b1321c56f2"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "fig, ax = plt.subplots(nrows=1, ncols=2, figsize=(18,3))\n",
    "\n",
    "df_train['dialog_len'] = df_train[['context_2', 'context_1', 'context_0']].apply(lambda x: 3-sum(x=='-'), axis=1)\n",
    "sns.countplot(x='dialog_len', data=df_train, ax=ax[0]);\n",
    "ax[0].set_title('train dialog length')\n",
    "\n",
    "df_public['dialog_len'] = df_public[['context_2', 'context_1', 'context_0']].apply(lambda x: 3-sum(x=='-'), axis=1)\n",
    "sns.countplot(x='dialog_len', data=df_public, ax=ax[1]);\n",
    "ax[1].set_title('public dialog length')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rvEIVOYqNx_e"
   },
   "source": [
    "<b>Опять таки мы видим, что распределения в трейне и тесте одинаковы\n",
    "\n",
    "<b>Далее проверяем граничные значения (Например, одно слово в context или reply), а именно: \n",
    "- распределение confidence и лейблов для длины диалога = 1\n",
    "- распределение confidence и лейблов для длины reply = 1\n",
    "- распределение confidence и лейблов для длины reply = 2\n",
    "- распределение confidence и лейблов для длины reply >= 2\n",
    "- распределение confidence и лейблов для длины reply = 1 и длины диалога = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 924
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4000,
     "status": "ok",
     "timestamp": 1524240949200,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "Gbjn02zqNx_f",
    "outputId": "d2a5a845-a58f-4c49-8668-63650f38e1a6"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "fig, ax = plt.subplots(nrows=1, ncols=2, figsize = (15,2))\n",
    "sns.distplot(df_train[df_train['dialog_len'] == 1].confidence, bins=100, kde=False, ax=ax[0])\n",
    "sns.countplot(x='label', data=df_train[df_train['dialog_len'] == 1], ax=ax[1])\n",
    "fig.suptitle('train dialog length==1 statistics')\n",
    "\n",
    "fig, ax = plt.subplots(nrows=1, ncols=2, figsize = (15,2))\n",
    "sns.distplot(df_train[df_train.reply.apply(lambda x: len(x.split()) == 1)].confidence, bins=100, kde=False, ax=ax[0])\n",
    "sns.countplot(x='label', data=df_train[df_train.reply.apply(lambda x: len(x.split()) == 1)], ax=ax[1])\n",
    "fig.suptitle('train reply length==1 statistics')\n",
    "\n",
    "fig, ax = plt.subplots(nrows=1, ncols=2, figsize = (15,2))\n",
    "sns.distplot(df_train[df_train.reply.apply(lambda x: len(x.split()) == 2)].confidence, bins=100, kde=False, ax=ax[0])\n",
    "sns.countplot(x='label', data=df_train[df_train.reply.apply(lambda x: len(x.split()) == 2)], ax=ax[1])\n",
    "fig.suptitle('train reply length==2 statistics')\n",
    "\n",
    "fig, ax = plt.subplots(nrows=1, ncols=2, figsize = (15,2))\n",
    "sns.distplot(df_train[df_train.reply.apply(lambda x: len(x.split()) >= 2)].confidence, bins=100, kde=False, ax=ax[0])\n",
    "sns.countplot(x='label', data=df_train[df_train.reply.apply(lambda x: len(x.split()) > 2)], ax=ax[1])\n",
    "fig.suptitle('train reply length>=2 statistics')\n",
    "\n",
    "fig, ax = plt.subplots(nrows=1, ncols=2, figsize = (15,2))\n",
    "sns.distplot(df_train[df_train.reply.apply(lambda x: len(x.split()) == 1) & (df_train['dialog_len'] == 1)].confidence, bins=100, kde=False, ax=ax[0])\n",
    "sns.countplot(x='label', data=df_train[df_train.reply.apply(lambda x: len(x.split()) == 1) & (df_train['dialog_len'] == 1)], ax=ax[1])\n",
    "fig.suptitle('train reply length==1 & dialog_len==1 statistics')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ClFYGvRUNx_m"
   },
   "source": [
    "<b>Как мы видим при длине диалога 1 и длине reply 1 у нас почти в большинстве случаев лейбл 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6GdWcPlQNx_n"
   },
   "source": [
    "<b>Дальше попытаемся как-то использовать confidence. В данном случае confidence для других лейблов считался как (1-confidence)/2 и для этой формулы confidence для тестовой части мы взяли как 1. В этом случае confidence для лейблов для тестовой части будет 0. Можно было избежать мороки с confidence для тестовой части 1 просто сделав срез для трейн части, но суть от этого не изменится. <br>\n",
    "\n",
    "В итоге мы расширили фичу confidence и будем их использовать как лейбл"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11685,
     "status": "ok",
     "timestamp": 1524240960924,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "fMYXRjk_Nx_q",
    "outputId": "e7cef74f-c15a-42a8-d4da-8c879ec696c0"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "df_data['inv_confidence_div2'] = (1 - df_data.confidence)/2\n",
    "\n",
    "df_data['label0_confidence'] = 0\n",
    "df_data['label1_confidence'] = 0\n",
    "df_data['label2_confidence'] = 0\n",
    "\n",
    "df_data['label0_confidence'] = df_data[['label0_confidence', 'label', 'inv_confidence_div2', 'confidence']].apply(lambda x: x.confidence if x.label==0 else x.inv_confidence_div2 ,axis=1)\n",
    "df_data['label1_confidence'] = df_data[['label1_confidence', 'label', 'inv_confidence_div2', 'confidence']].apply(lambda x: x.confidence if x.label==1 else x.inv_confidence_div2 ,axis=1)\n",
    "df_data['label2_confidence'] = df_data[['label2_confidence', 'label', 'inv_confidence_div2', 'confidence']].apply(lambda x: x.confidence if x.label==2 else x.inv_confidence_div2 ,axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "PkGSV6QxNx_u"
   },
   "source": [
    "<b>Теперь нужно избавиться от всех символов кроме русских (в датасете могут присутствовать не русские символы и на самом деле так и есть) , так как наш эмбеддинг над словами будет из fasttext. Сделаем это с помощью регулярных выражений"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1378,
     "status": "ok",
     "timestamp": 1524240962336,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "8My6M8qMNx_w",
    "outputId": "ace73d32-4f77-42c3-c30e-5ad7308f4654"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "reg = re.compile('[^а-яА-Я]')\n",
    "df_data['context_2'] = df_data['context_2'].apply(lambda x: reg.sub(r' ', x))\n",
    "df_data['context_1'] = df_data['context_1'].apply(lambda x: reg.sub(r' ', x))\n",
    "df_data['context_0'] = df_data['context_0'].apply(lambda x: reg.sub(r' ', x))\n",
    "df_data['reply'] = df_data['reply'].apply(lambda x: reg.sub(r' ', x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xQ9E-9HxNx_1"
   },
   "source": [
    "<b>Теперь возьмем слова с длиной 3 символа и более (потому что наш эмбеддинг обучался на 3-5 грамах) и обрежем наши context и reply. Для того, чтобы понять на сколько обрезать нужно смотреть на наши графики с распределением кол-ва слов. В итоге имеем: context2 - 10 слов, context1 - 15 слов, context0 - 20 слов, reply - 15 слов."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3703,
     "status": "ok",
     "timestamp": 1524240966115,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "BKgf1ll5Nx_4",
    "outputId": "012b3b7c-ab6c-4f4b-e460-bfe5639a2e88"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "temp = df_data['context_2'].apply(lambda x: x.split(' '))\n",
    "df_data['context_2_cutted'] = temp.apply( lambda x: list(filter(lambda y : len(y) >= 3, x))[:10] )\n",
    "\n",
    "temp = df_data['context_1'].apply(lambda x: x.split(' '))\n",
    "df_data['context_1_cutted'] = temp.apply(lambda x: list(filter(lambda y : len(y) >= 3, x))[:15] )\n",
    "\n",
    "temp = df_data['context_0'].apply(lambda x: x.split(' '))\n",
    "df_data['context_0_cutted'] = temp.apply(lambda x: list(filter(lambda y : len(y) >= 3, x))[:20] )\n",
    "\n",
    "temp = df_data['reply'].apply(lambda x: x.split(' '))\n",
    "df_data['reply_cutted'] = temp.apply(lambda x: list(filter(lambda y : len(y) >= 3, x))[:15] )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ee3ym5k1Nx_9"
   },
   "source": [
    "<b>Найдем и удалим из данных строки, где после наших махинаций длина reply или context0 равны 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1863,
     "status": "ok",
     "timestamp": 1524241044397,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "H8RZ_UmgNx__",
    "outputId": "d3a0d2f9-459d-47c5-9a6e-66a58b29e7d6"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "data_zero_reply_loc = df_data['reply_cutted'][df_data['reply_cutted'].apply(lambda x: len(x)==0)].index.values\n",
    "data_zero_context_0_loc = df_data['context_0_cutted'][df_data['context_0_cutted'].apply(lambda x: len(x)==0)].index.values\n",
    "\n",
    "data_noise = list(set(data_zero_reply_loc) | set(data_zero_context_0_loc))\n",
    "df_data_cleared = df_data.drop(labels=data_noise)\n",
    "print(len(data_noise))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 181
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1990,
     "status": "ok",
     "timestamp": 1524241046596,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "rW7Y-g9pNyAG",
    "outputId": "af5d9a81-f3db-4611-8845-7cb99080d489"
   },
   "outputs": [],
   "source": [
    "df_data_cleared.sample()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-hF1nZMHNyAJ"
   },
   "source": [
    "# Часть кросс валидации\n",
    "<b>Теперь разберемся с кроссвалидацией и моделью. <br>\n",
    "Модель выбрана нейросетевая (хотя она тут не очень подходит, так как данных для нейронной сети (глубокой) очень мало, порядка 70 тыс.) <br>\n",
    "Валидационная выборка это будут последние 20% теста. Данный тип валидации подходит, так как здесь нет временной зависимости или какой-либо еще не тривиальной.\n",
    "\n",
    "Модель: 2 входа context и reply, далее embedding на fasttext, CNN для context и для reply. Потом идет конкатенация двух CNN, потом Dense слой и слой softmax. loss - categorical_crossentropy, оптимизатор - AMSgrad."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "9IGdYXtgNyAK"
   },
   "outputs": [],
   "source": [
    "df_train = df_data_cleared.loc[:train_tres]\n",
    "CV_COEF = 0.8\n",
    "cv_train_tres = int(len(df_train) * CV_COEF)\n",
    "cv_train_y = df_train[['label0_confidence', 'label1_confidence', 'label2_confidence']][:cv_train_tres].values\n",
    "cv_val_y = df_train[['label0_confidence', 'label1_confidence', 'label2_confidence']][cv_train_tres:].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "gyJxsRdQNyAO"
   },
   "source": [
    "<b>Создадим данные для входа в нейросеть (конатенация диалога без reply)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "PAb2dgm8NyAO"
   },
   "outputs": [],
   "source": [
    "cv_train_context = df_train['context_2_cutted'] + df_train['context_1_cutted'] + df_train['context_0_cutted']\n",
    "cv_train_reply = df_train['reply_cutted']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kNk2GSWwNyAR"
   },
   "source": [
    "<b>Определим максимальную длину диалога для каждого входа, размер эмбеддинга (задается самой моделью fasttext-а) и кол-во токенов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "o3EbkCuRNyAT"
   },
   "outputs": [],
   "source": [
    "MAX_NB_WORDS = 100000\n",
    "MAX_SEQUENCE_LENGTH_CONTEXT = 45\n",
    "MAX_SEQUENCE_LENGTH_REPLY = 15\n",
    "EMBEDDING_DIM = 300"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MewzcXgONyAW"
   },
   "source": [
    "<b>Создадим токены, приведем входные данные к одной длине и разобьем данные на трейн и валидацию"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4049,
     "status": "ok",
     "timestamp": 1524241226655,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "CZtgHpAPNyAW",
    "outputId": "75f76994-bd50-43bd-9353-0f746cba51d7"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "tokenizer_context = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer_context.fit_on_texts(cv_train_context)\n",
    "sequences_context = tokenizer_context.texts_to_sequences(cv_train_context)\n",
    "word_index_context = tokenizer_context.word_index\n",
    "data_context = pad_sequences(sequences_context, maxlen=MAX_SEQUENCE_LENGTH_CONTEXT)\n",
    "\n",
    "X_train_context = data_context[:cv_train_tres]\n",
    "X_val_context = data_context[cv_train_tres:]\n",
    "\n",
    "#################################################################################\n",
    "\n",
    "tokenizer_reply = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer_reply.fit_on_texts(cv_train_reply)\n",
    "sequences_reply = tokenizer_reply.texts_to_sequences(cv_train_reply)\n",
    "word_index_reply = tokenizer_reply.word_index\n",
    "data_reply = pad_sequences(sequences_reply, maxlen=MAX_SEQUENCE_LENGTH_REPLY)\n",
    "\n",
    "X_train_reply = data_reply[:cv_train_tres]\n",
    "X_val_reply = data_reply[cv_train_tres:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "C3PRqgnmNyAa"
   },
   "source": [
    "<b>Создадим эмбеддинги для слов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 4103,
     "status": "ok",
     "timestamp": 1524241230807,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "p6cNzuV9NyAb",
    "outputId": "a9dfd299-0410-480a-aab0-9bc73d3075da"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "input_context = Input(shape=(MAX_SEQUENCE_LENGTH_CONTEXT, ))\n",
    "embedding_matrix_context = np.zeros((len(word_index_context) + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index_context.items():\n",
    "    embedding_vector = ft.wv[word]\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix_context[i] = embedding_vector\n",
    "word_embed_context = Embedding(len(word_index_context) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix_context],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH_CONTEXT,\n",
    "                            trainable=False)(input_context)\n",
    "\n",
    "##############################################################################\n",
    "\n",
    "input_reply = Input(shape=(MAX_SEQUENCE_LENGTH_REPLY, ))\n",
    "embedding_matrix_reply = np.zeros((len(word_index_reply) + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index_reply.items():\n",
    "    embedding_vector = ft.wv[word]\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix_reply[i] = embedding_vector\n",
    "word_embed_reply = Embedding(len(word_index_reply) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix_reply],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH_REPLY,\n",
    "                            trainable=False)(input_reply)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4b6UKXNDNyAf"
   },
   "source": [
    "<b>Определим модель и начнем обучение на 10 эпохах с батчем в 64. <br>\n",
    "  Парамерты и архитектура модели подбирались на основе валидации путем проб и ошибок."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 1785
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 224876,
     "status": "ok",
     "timestamp": 1524253069907,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "2pcLb5rgNyAg",
    "outputId": "3b5eac53-6c66-446f-9065-a2c33aed911b"
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 64\n",
    "\n",
    "context_conv1 = Conv1D(16, 5, padding='same')(word_embed_context)\n",
    "context_max_pool1 = MaxPooling1D(pool_size=2)(context_conv1)\n",
    "context_conv2 = Conv1D(32, 3, padding='same')(context_max_pool1)\n",
    "context_max_pool2 = MaxPooling1D(pool_size=2)(context_conv2)\n",
    "context_conv3 = Conv1D(64, 3, padding='same')(context_max_pool2)\n",
    "context_max_pool3 = MaxPooling1D(pool_size=2)(context_conv3)\n",
    "context_flatten = Flatten()(context_max_pool3)\n",
    "\n",
    "reply_conv1 = Conv1D(32, 5, padding='same')(word_embed_reply)\n",
    "reply_max_pool1 = MaxPooling1D(pool_size=2)(reply_conv1)\n",
    "reply_conv2 = Conv1D(64, 3, padding='same')(reply_max_pool1)\n",
    "reply_max_pool2 = MaxPooling1D(pool_size=2)(reply_conv2)\n",
    "reply_flatten = Flatten()(reply_max_pool2)\n",
    "\n",
    "merged = concatenate([context_flatten, reply_flatten])\n",
    "drop_0 = Dropout(0.3)(merged)\n",
    "dense_0 = Dense(256, activation='sigmoid')(drop_0)\n",
    "drop_1 = Dropout(0.3)(dense_0)\n",
    "dense_out = Dense(3, activation='softmax')(drop_1)\n",
    "\n",
    "model = Model(inputs=[input_context, input_reply], outputs=[dense_out])\n",
    "model.compile(optimizer=optimizers.Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "print(model.summary())\n",
    "\n",
    "hist = model.fit([X_train_context, X_train_reply], cv_train_y, batch_size=BATCH_SIZE, epochs=10, validation_data=([X_val_context, X_val_reply], cv_val_y))\n",
    "plt.figure()\n",
    "plt.title('acc')\n",
    "plt.plot(hist.history['val_acc'], label='val')\n",
    "plt.plot(hist.history['acc'], label='train')\n",
    "plt.legend()\n",
    "plt.figure()\n",
    "plt.title('loss')\n",
    "plt.plot(hist.history['val_loss'], label='val')\n",
    "plt.plot(hist.history['loss'], label='train')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Uxw2LrHU58C7"
   },
   "source": [
    "<b> Теперь обучим модель на 6 эпохах (выбор был сделан исходя из графика лосса на валидации - минимальная точка на графике)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 271
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 136273,
     "status": "ok",
     "timestamp": 1524253249178,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "1RttjL7Y4Ar_",
    "outputId": "d15a7663-9ebd-4006-f50c-f0948353f15f"
   },
   "outputs": [],
   "source": [
    "context_conv1 = Conv1D(16, 5, padding='same')(word_embed_context)\n",
    "context_max_pool1 = MaxPooling1D(pool_size=2)(context_conv1)\n",
    "context_conv2 = Conv1D(32, 3, padding='same')(context_max_pool1)\n",
    "context_max_pool2 = MaxPooling1D(pool_size=2)(context_conv2)\n",
    "context_conv3 = Conv1D(64, 3, padding='same')(context_max_pool2)\n",
    "context_max_pool3 = MaxPooling1D(pool_size=2)(context_conv3)\n",
    "context_flatten = Flatten()(context_max_pool3)\n",
    "\n",
    "reply_conv1 = Conv1D(32, 5, padding='same')(word_embed_reply)\n",
    "reply_max_pool1 = MaxPooling1D(pool_size=2)(reply_conv1)\n",
    "reply_conv2 = Conv1D(64, 3, padding='same')(reply_max_pool1)\n",
    "reply_max_pool2 = MaxPooling1D(pool_size=2)(reply_conv2)\n",
    "reply_flatten = Flatten()(reply_max_pool2)\n",
    "\n",
    "merged = concatenate([context_flatten, reply_flatten])\n",
    "drop_0 = Dropout(0.3)(merged)\n",
    "dense_0 = Dense(256, activation='sigmoid')(drop_0)\n",
    "drop_1 = Dropout(0.3)(dense_0)\n",
    "dense_out = Dense(3, activation='softmax')(drop_1)\n",
    "\n",
    "model = Model(inputs=[input_context, input_reply], outputs=[dense_out])\n",
    "model.compile(optimizer=optimizers.Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "model.fit([X_train_context, X_train_reply], cv_train_y, batch_size=BATCH_SIZE, epochs=6, validation_data=([X_val_context, X_val_reply], cv_val_y))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3Vl9ROhqT4t4"
   },
   "source": [
    "<b> Делаем предикт для валидационной выборке и считаем скор"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3072,
     "status": "ok",
     "timestamp": 1524253290300,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "fHTAOMbbTO35",
    "outputId": "c163db00-1aaa-4b7e-c672-628c5778f926"
   },
   "outputs": [],
   "source": [
    "y_pred = model.predict([X_val_context, X_val_reply])\n",
    "y_pred_classes = np.argmax(y_pred, axis=1)\n",
    "cv_val_classes = np.argmax(cv_val_y, axis=1)\n",
    "nDCG(cv_val_classes, y_pred_classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "tjZq8oYZNyAn"
   },
   "source": [
    "# Часть обучения\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "jVQk1a6dNyA0"
   },
   "source": [
    "<b> Отделим тест и трейн"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1552,
     "status": "ok",
     "timestamp": 1524253320412,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "IJA_4XXbNyA1",
    "outputId": "fb63b95b-b0a4-4d70-a8e2-21505fab32d6"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "train_inds = df_data[df_data.label != '-'].index.values\n",
    "test_inds = df_data[df_data.label == '-'].index.values\n",
    "y_train = df_data[df_data.label != '-'][['label0_confidence', 'label1_confidence', 'label2_confidence']].values\n",
    "\n",
    "context = df_data['context_2_cutted'] + df_data['context_1_cutted'] + df_data['context_0_cutted']\n",
    "reply = df_data['reply_cutted']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "quHI33U4TD7q"
   },
   "source": [
    "<b>Сделаем те же процедуры для эмбединга, какие делали для кроссвалидации \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3482,
     "status": "ok",
     "timestamp": 1524253323928,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "r-by9a3mQaLn",
    "outputId": "ca870221-b612-46af-80b4-d222cf1c60c9"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "tokenizer_context = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer_context.fit_on_texts(context)\n",
    "sequences_context = tokenizer_context.texts_to_sequences(context)\n",
    "word_index_context = tokenizer_context.word_index\n",
    "data_context = pad_sequences(sequences_context, maxlen=MAX_SEQUENCE_LENGTH_CONTEXT)\n",
    "\n",
    "train_context = data_context[train_inds]\n",
    "test_context = data_context[test_inds]\n",
    "\n",
    "#################################################################################\n",
    "\n",
    "tokenizer_reply = Tokenizer(num_words=MAX_NB_WORDS)\n",
    "tokenizer_reply.fit_on_texts(reply)\n",
    "sequences_reply = tokenizer_reply.texts_to_sequences(reply)\n",
    "word_index_reply = tokenizer_reply.word_index\n",
    "data_reply = pad_sequences(sequences_reply, maxlen=MAX_SEQUENCE_LENGTH_REPLY)\n",
    "\n",
    "train_reply = data_reply[train_inds]\n",
    "test_reply = data_reply[test_inds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 11080,
     "status": "ok",
     "timestamp": 1524253335060,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "dpVA192PQdA4",
    "outputId": "f1a38138-d7be-4c7e-b1cf-fdad793a8535"
   },
   "outputs": [],
   "source": [
    "%%time\n",
    "\n",
    "input_context = Input(shape=(MAX_SEQUENCE_LENGTH_CONTEXT, ))\n",
    "embedding_matrix_context = np.zeros((len(word_index_context) + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index_context.items():\n",
    "    embedding_vector = ft.wv[word]\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix_context[i] = embedding_vector\n",
    "word_embed_context = Embedding(len(word_index_context) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix_context],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH_CONTEXT,\n",
    "                            trainable=False)(input_context)\n",
    "\n",
    "##############################################################################\n",
    "\n",
    "input_reply = Input(shape=(MAX_SEQUENCE_LENGTH_REPLY, ))\n",
    "embedding_matrix_reply = np.zeros((len(word_index_reply) + 1, EMBEDDING_DIM))\n",
    "for word, i in word_index_reply.items():\n",
    "    embedding_vector = ft.wv[word]\n",
    "    if embedding_vector is not None:\n",
    "        embedding_matrix_reply[i] = embedding_vector\n",
    "word_embed_reply = Embedding(len(word_index_reply) + 1,\n",
    "                            EMBEDDING_DIM,\n",
    "                            weights=[embedding_matrix_reply],\n",
    "                            input_length=MAX_SEQUENCE_LENGTH_REPLY,\n",
    "                            trainable=False)(input_reply)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "AxanU1aPNyBF"
   },
   "source": [
    "<b> Обучим сеть на всем тренировочном датасете с количеством эпох в 6\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 1107
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 169021,
     "status": "ok",
     "timestamp": 1524253683274,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "SmljsBZmQrf9",
    "outputId": "e0a5f0f5-e536-41ba-ea4b-eb654b0f6a51"
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 64\n",
    "\n",
    "context_conv1 = Conv1D(16, 5, padding='same')(word_embed_context)\n",
    "context_max_pool1 = MaxPooling1D(pool_size=2)(context_conv1)\n",
    "context_conv2 = Conv1D(32, 3, padding='same')(context_max_pool1)\n",
    "context_max_pool2 = MaxPooling1D(pool_size=2)(context_conv2)\n",
    "context_conv3 = Conv1D(64, 3, padding='same')(context_max_pool2)\n",
    "context_max_pool3 = MaxPooling1D(pool_size=2)(context_conv3)\n",
    "context_flatten = Flatten()(context_max_pool3)\n",
    "\n",
    "reply_conv1 = Conv1D(32, 3, padding='same')(word_embed_reply)\n",
    "reply_max_pool1 = MaxPooling1D(pool_size=2)(reply_conv1)\n",
    "reply_conv2 = Conv1D(64, 3, padding='same')(reply_max_pool1)\n",
    "reply_max_pool2 = MaxPooling1D(pool_size=2)(reply_conv2)\n",
    "reply_flatten = Flatten()(reply_max_pool2)\n",
    "\n",
    "merged = concatenate([context_flatten, reply_flatten])\n",
    "drop_0 = Dropout(0.3)(merged)\n",
    "dense_0 = Dense(256, activation='sigmoid')(drop_0)\n",
    "drop_1 = Dropout(0.3)(dense_0)\n",
    "dense_out = Dense(3, activation='softmax')(drop_1)\n",
    "\n",
    "model = Model(inputs=[input_context, input_reply], outputs=[dense_out])\n",
    "model.compile(optimizer=optimizers.Adam(lr=0.0001), loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "print(model.summary())\n",
    "\n",
    "model.fit([train_context, train_reply], y_train, batch_size=BATCH_SIZE, epochs=6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "kqX40MJ1NyBK"
   },
   "source": [
    "<b> Сделаем предикт по тесту и сохраним в csv файле"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "tBl53xqvNyBK"
   },
   "outputs": [],
   "source": [
    "y_pred = model.predict([test_context, test_reply])\n",
    "y_pred_classes = np.argmax(y_pred, axis=1)\n",
    "y_pred_prob = np.max(y_pred, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     },
     "base_uri": "https://localhost:8080/",
     "height": 218
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2128,
     "status": "ok",
     "timestamp": 1524253704799,
     "user": {
      "displayName": "Алексей Иванов",
      "photoUrl": "https://lh3.googleusercontent.com/a/default-user=s128",
      "userId": "104814834381580890532"
     },
     "user_tz": -180
    },
    "id": "16umXAXQNyBU",
    "outputId": "42817618-b77a-436c-801e-a6d9a8ff582c"
   },
   "outputs": [],
   "source": [
    "df_public['label'] = y_pred_classes\n",
    "df_public['confidence'] = y_pred_prob\n",
    "df_public.sort_values(by=['context_id', 'confidence'], ascending=False)[['context_id', 'reply_id']].to_csv('subm.csv', encoding='utf-8', sep=' ', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1K7WLgChVhTB"
   },
   "source": [
    "<b> Скачаем csv файл с колаба"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "autoexec": {
      "startup": false,
      "wait_interval": 0
     }
    },
    "colab_type": "code",
    "id": "hVEy5DrQVBWh"
   },
   "outputs": [],
   "source": [
    "from google.colab import files\n",
    "files.download('subm.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nvqG5JecWy0t"
   },
   "source": [
    "<b> Скор 83919, что соответствует ожиданиям исходя скору на валидации"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "default_view": {},
   "name": "Base.ipynb",
   "provenance": [],
   "version": "0.3.2",
   "views": {}
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
